#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@File    :   ip_info.py
@Contact :   291622538@qq.com

@Modify Time      @Author    @Version    @Description
------------      -------    --------    -----------
2020/11/9 14:52   fan        1.0         None
"""
import csv
import datetime
import json

from scrapy import Spider
from scrapy import Request

from website.items import IpInfoItem, IpInfoOverItem


class IpInfoGet(Spider):
    # 定义爬虫名称
    name = "ip_info_get"

    custom_settings = {
        'ITEM_PIPELINES': {'website.pipelines.SaveIpInfoPipeline': 300},
    }

    def start_requests(self):

        with open('./data/ip_data/1.csv', 'r', encoding='utf-8') as f:
            reader = csv.reader(f)
            for num, line in enumerate(reader):
                # if num == 20:
                #     break
                domainIp = line[0]
                URL = "https://pro.ip-api.com/json/%s?key=MWtfDmIJu97YVlY&fields=66846719&lang=zh-CN" % domainIp
                yield Request(URL, callback=self.original_parse, meta={"domainIp": domainIp},
                              dont_filter=False,
                              errback=lambda response, domainIp=domainIp: self.save_overtime(
                                  response, domainIp))

    def original_parse(self, response, **kwargs):
        domainInfo = response.text
        try:
            ipInfo = json.loads(domainInfo)
            ipInfoItem = IpInfoItem()
            ipInfoItem["domainIp"] = response.meta["domainIp"]
            ipInfoItem["continent"] = ipInfo["continent"]
            ipInfoItem["continentCode"] = ipInfo["continentCode"]
            ipInfoItem["country"] = ipInfo["country"]
            ipInfoItem["countryCode"] = ipInfo["countryCode"]
            ipInfoItem["region"] = ipInfo["region"]
            ipInfoItem["regionName"] = ipInfo["regionName"]
            ipInfoItem["city"] = ipInfo["city"]
            ipInfoItem["district"] = ipInfo["district"]
            ipInfoItem["zip"] = ipInfo["zip"]
            ipInfoItem["latitude"] = ipInfo["lat"]
            ipInfoItem["longitude"] = ipInfo["lon"]
            ipInfoItem["timezone"] = ipInfo["timezone"]
            ipInfoItem["offset"] = ipInfo["offset"]
            ipInfoItem["currency"] = ipInfo["currency"]
            ipInfoItem["isp"] = ipInfo["isp"]
            ipInfoItem["org"] = ipInfo["org"]
            ipInfoItem["ASNUM"] = ipInfo["as"]
            ipInfoItem["asName"] = ipInfo["asname"]
            ipInfoItem["reverse"] = ipInfo["reverse"]
            ipInfoItem["mobile"] = ipInfo["mobile"]
            ipInfoItem["proxy"] = ipInfo["proxy"]
            ipInfoItem["hosting"] = ipInfo["hosting"]
            ipInfoItem["addDate"] = datetime.datetime.now().strftime('%Y-%m-%d')

            yield ipInfoItem
        except Exception as e:
            print(e)
            over_domain = IpInfoOverItem(domainIp=response.meta["domainIp"])
            yield over_domain


    def save_overtime(self, response, domainIp):
        over_domain = IpInfoOverItem(domainIp=domainIp)
        yield over_domain
